from re_common.baselibrary.utils.core.mdeprecated import try_except2_async, retry_func_async

from apps.crawler_platform.core_callback.err_callback import all_client_back_callback
from apps.crawler_platform.core_platform.base_down import BaseCoreStep
from apps.crawler_platform.core_platform.core_nosql import NoSqlUtil


class BaseETL(BaseCoreStep):

    @retry_func_async(retry_times=3, sleep_time=2)
    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def read_nosql_html(self):
        """
        存储到nosql数据库
        """
        # 期刊列表级默认保存历史
        bools, dicts = await NoSqlUtil.read_nosql_html(self.pm)
        if not bools:
            return bools, dicts
        data = self.pm.mongo_html["data"]
        # 如果mongo查出来不存在
        if not data or not data[0]["last_data"]:
            return False, {"err_msg": "qk article local etl mongoDB里journal_article未查询到数据;" + str(self.pm.get_pkm())}
        data = data[0]["last_data"]
        try:
            del data["down_date"]
            del data["down_time"]
        except:
            pass
        if self.pm.task_name == 'cnkijournal' and self.pm.task_tag == 'cnkiarticle' and "2_1" in data["down_dict"]:
            del data["down_dict"]["2_1"]
        self.pm.down_model = self.pm.down_model.set_parse_obj(data)
        return True, {}

    def get_html(self, *args, **kwargs):
        """
        获取html数据，加载到model中
        """
        pass

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def get_redis(self):
        """
        获取redis的缓存
        """
        bools, err_info = await super().get_redis()
        if not bools:
            return bools, err_info
        from app_main import app
        self.pm.etl_redis_journal_rawids = await app.state.redis.hgetall("journal_rawids")
        return bools, err_info

    @retry_func_async(retry_times=3, sleep_time=2)
    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def save_nosql_html_etl(self):
        """
        ETL存储到nosql数据库
        """
        if self.pm.deal_model.status != "FAILED":
            return await NoSqlUtil.save_nosql_html_etl(self.pm)
        return True, {}
